package spark.MLlib;

import com.clearspring.analytics.util.Lists;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.stat.MultivariateStatisticalSummary;
import org.apache.spark.mllib.stat.Statistics;

import java.util.List;

/**
 * 作者: LDL
 * 功能说明:
 * 创建日期: 2015/6/30 14:32
 */
public class BasicStatistics {

    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setMaster("local").setAppName("JavaDataTypes");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        List<Vector> vectors = Lists.newArrayList();
        vectors.add(Vectors.dense(1,1,1));
        vectors.add(Vectors.dense(2,2,2));
        vectors.add(Vectors.dense(3,3,3));
        System.out.println(vectors);
        JavaRDD<Vector> mat = jsc.parallelize(vectors);
        MultivariateStatisticalSummary summary = Statistics.colStats(mat.rdd());
        System.out.println(summary.mean()); // a dense vector containing the mean value for each column
        System.out.println(summary.variance()); // column-wise variance
        System.out.println(summary.numNonzeros()); // number of nonzeros in each column
    }
}
